#include <cstdio>
#include <cstdlib>
#include <cstring>
#include <dirent.h>
#include <iostream>
#include <sstream>
#include <fstream>
#include <string>
#include <set>
#include <map>
#include <queue>
#include <cmath>
#include <db.h>
#include <errno.h>
#include <time.h>
#include <sys/time.h>
#include <sys/types.h>
#include <netinet/in.h>
#include <sys/socket.h>
#include <sys/wait.h>
#include <arpa/inet.h>
#include <libgen.h>
#include <algorithm>

using namespace std;

typedef struct pos_num{ //存放每个文件中关键字的信息
        int pos; //第一次出现的行数
        int tf; 
}POS_NUM;

typedef struct keynum //数据库中数据项
{
        int docid;
        int tf;
        int pos;
}KEYNUM;


const int MAX_FILE_NUM = 10000; //输入文件最大数目
const int MAX_KEY_NUM = 100; //一次询问中关键词的最大个数
char question_buf[100]; // 询问的问题
char answer_buf[10000]; //每次询问的答案
int answer_ind;//答案的长度
string strfile[MAX_FILE_NUM];  //存放输入文件名
string strkey[MAX_KEY_NUM]; //存放每一次询问中关键字
POS_NUM mat[MAX_FILE_NUM][MAX_KEY_NUM]; //记录关键字，文件的二元组
POS_NUM mmat[MAX_FILE_NUM][MAX_KEY_NUM]; //记录附加数据库中关键字，文件的二元组
int num[MAX_KEY_NUM]; //记录每个关键字在总文件中出现的次数
string insstrfile[MAX_FILE_NUM];
int insfileind;
DB *dbp;      //数据库
DBT key, data; 
DBC *cursorp;
DB *dbpp;
DBC *cursorpp;
int keyind = 0;
int fileind = 0;


set<string> keyst; //存放字典的set
set<string> delfile; //存放删除文件名
map<string,int> mapkey; //存放分词结果
map<string,int> mapkey2;//暂时存放正向最大匹配的分词
map<string,int> mapkey3;//暂时存放逆向最大匹配的分词
int len2; //存放正向最大匹配的分词个数
int len3; //逆向
int cnt2; //对正向最大匹配中单个字的个数
int cnt3; //逆向

const char *mainfilename = "../filename/mainfilename.txt";
const char *delfilename = "../filename/delfilename.txt";
const char *insfilename = "../filename/insfilename.txt";
const char *sourcedir = "../source/";
const char *dicfile = "../dict/words_gbk.dic";
const char *mainindexdb = "../db/mainindex.db";
const char *auxindexdb = "../db/auxindex.db";

long long GetMillSec()
{
	long long nMillSec = 0;

	struct timeval tv; 
	gettimeofday(&tv,NULL);

	nMillSec = (long long)tv.tv_sec * 1000;
	//cout << "nMIllSec1 = " << nMillSec << endl;

	nMillSec += tv.tv_usec / 1000;
	//cout << "nMIllSec2 = " << nMillSec << endl;

	return nMillSec;
}

void readfilename() //读取原有文件名，新增文件名，删除文件名
{
        ifstream infile(mainfilename,ifstream::in);
        string strtmp;
        fileind = 0;
	while(getline(infile,strtmp,'\n')){ 
	        strfile[fileind++] = strtmp;

        }
        
        infile.close();
        ifstream infile2(insfilename,ifstream::in);
        insfileind = 0;
	while(getline(infile2,strtmp,'\n')){ 
	        insstrfile[insfileind++] = strtmp;
        }
        infile2.close();
        ifstream infile3(delfilename,ifstream::in);
        delfile.clear();
	while(getline(infile3,strtmp,'\n')){ //取出其中一行进行分词处理
	        delfile.insert(strtmp);
        }
        infile3.close();

        
}

void builddict()  //创建字典
{
        ifstream infile(dicfile,ifstream::in);
        if(!infile.is_open()){
		printf("open inputfile error!\n");
		return ;
	}
	int num;
	string keys;
	while(infile>>num>>keys){
	       keyst.insert(keys);
        }
        infile.close();
}

bool find(string str)  //查找关键词是否存在
{
	if(keyst.find(str)!=keyst.end())
		return true;
	else
		return false;
}

bool stoplisten(char ch)  //英文停用词表
{
        return ch=='.'||ch==' '||ch==';'||ch=='	';
}

string strtolower(string str)
{
       for(int i=0;i<str.length();i++){
            str[i] = tolower(str[i]);
       }
       return str;
}

void segmenten(string enstr)  //将英文串分词
{
        int ind = 0;
        while(enstr[ind]&&stoplisten(enstr[ind])) ind++;
        if(!enstr[ind])
                return;
        bool flag = false;
        for(int i=ind;enstr[i];i++){
                if(!flag&&stoplisten(enstr[i])){
                       flag=true;
                       ++mapkey[strtolower(enstr.substr(ind,i-ind))];
                       ind = i;
                }
                else if(flag&&!stoplisten(enstr[i])){
                       flag = false;
                       ind = i;
                }
        }
        int len = enstr.length();
        ++mapkey[strtolower(enstr.substr(ind,len-ind))];
}




void segmentch(string enstr,bool flag) //将中文串分词
{      
        int len = enstr.length();
        int clen;
        string st;
        if(flag){
                while(!enstr.empty()){
                  st = enstr;
                  clen = len;
                  while(clen>2&&!find(st)){
                    clen -=1;  //若在字典找不到对应的关键字，则一个个输出
                    st = enstr.substr(len-clen,clen);
                  }
                  if(clen>0){
                    ++mapkey2[st];
                    enstr = enstr.substr(0,len-clen);
                  }     
                  len = enstr.length();          
               }
        }
        else{
               while(!enstr.empty()){
                   st = enstr;
                   clen = len;
                   while(clen>2&&!find(st)){
                       clen -= 1;
                       st = enstr.substr(0,clen);
                   }
                   if(clen>0){
                       ++mapkey3[st];
                       enstr = enstr.substr(clen,len-clen);
                   }
                   len = enstr.length();
               }
        }
}

bool comp()  //比较正向分词和逆向分词结果，若相同，return true,否则，return false
{
       if(mapkey2.size()!=mapkey3.size())
              return false;
       for(map<string,int>::iterator iter2 = mapkey2.begin(),iter3 = mapkey3.begin();iter2 != mapkey2.end() && iter3 != mapkey3.end(); iter2++,iter3++){
              if(iter2->first == iter3->first && iter2->second == iter3->second){
                    continue;
              }
              else
                    return false;
       }
       return true;
}

void segmentboth(string enstr) //进行逆向分词和正向分词的主控程序，若两者不相同，则比较两者分词个数，考虑分词个数少的那种，若相同，则考虑两种分词中单字的个数最少的，若仍相同，选择正向分词
{
        cnt2 = cnt3 = 0;
        len2 = len3 = 0;
        mapkey2.clear();
        mapkey3.clear();
        segmentch(enstr,0);
        segmentch(enstr,1);
        int flag = 1;
        if(comp()){
              flag = 1;
              
        }
        else{
              if(len2 < len3){
                     flag = 1;
              }
              else if(len2 > len3){
                     flag = -1;
              }
              else{
                     if(cnt2<cnt3)
                          flag = 1;
                     else if(cnt2>cnt3)
                          flag = -1;
                     else 
                          flag = 1;
              }
        }
        if(flag == 1){ //将正向分词结果并入
              for(map<string,int>::iterator iter = mapkey2.begin();iter != mapkey2.end();iter++){
                    map<string,int>::iterator iter0 =  mapkey.find(iter->first);
                    if(iter0==mapkey.end()){
                         mapkey.insert(pair<string,int>(iter->first,iter->second));
                    }
                    else
                         iter0->second += iter->second;
              }
        }
        else{ //将逆向分词结果并入
              for(map<string,int>::iterator iter = mapkey3.begin();iter != mapkey3.end();iter++){
                     map<string,int>::iterator iter0 =  mapkey.find(iter->first);
                     if(iter0==mapkey.end()){
                         mapkey.insert(pair<string,int>(iter->first,iter->second));
                     }
                     else
                         iter0->second += iter->second;
              }
        }
}

void segmentstring(string st) //st为输入字符串，最大逆序匹配 ：由于设定了关键字最大长度，对于中英文混合分词处理起来有一定的麻烦，这里只是将分隔的英文分开，最后再将中文分词
{
	string st2;
	int len = st.length();
	int clen ;
	mapkey.clear();
	puts("hello");
	while(!st.empty()){
	       clen = len;
	       st2 = st.substr(len-clen,len);
	       bool flag = (st2[0]>0)?1:0;
	       int ind = 0;
	       for(int i =0;i<clen;i++){
	             if(flag&&st2[i]<0){
	                   flag = 0;
	                   cout<<st2.substr(ind,i-ind)<<'#'<<endl;
	                   segmenten(st2.substr(ind,i-ind));
	                   ind = i;
                     }
                     else if(!flag&&st2[i]>0){
                           flag =1 ;
                           cout<<st2.substr(ind,i-ind)<<'*'<<endl;
                           segmentboth(st2.substr(ind,i-ind));
                           ind = i;
                     }
               }
	       if(st2[ind]>0){
	             cout<<st2.substr(ind,clen-ind)<<'#'<<endl;
	             segmenten(st2.substr(ind,clen-ind));
               }
               else{
                     cout<<st2.substr(ind,clen-ind)<<'*'<<endl;
                     segmentboth(st2.substr(ind,clen-ind));
               }
               st = st.substr(0,len-clen);
               len = st.length();
        }
        keyind = 0;
        for(map<string ,int >::iterator iter = mapkey.begin();iter!=mapkey.end();iter++){
               strkey[keyind++] = iter->first;
        }
}

void print_error(int ret)
{
	if(ret != 0)
		printf("ERROR: %s\n",db_strerror(ret));
}

int dbcmp(DB *dbp,const DBT *data1,const DBT *data2)
{
         return (((KEYNUM *)(data2->data))->tf)-(((KEYNUM *)(data1->data))->tf);
}

//DBT 初始化
void init_DBT(DBT * key, DBT * data)
{
	memset(key, 0, sizeof(DBT));
	memset(data, 0, sizeof(DBT));
}


void init_db(const char *dbname) //初始化主索引数据库
{
        int ret;
        ret = db_create(&dbp, NULL, 0);
	print_error(ret);
	ret = dbp->set_flags(dbp,DB_DUPSORT); 
	print_error(ret);
	//ret = dbp->set_dup_compare(dbp,dbcmp);
	//print_error(ret);
	ret = dbp->set_bt_compress(dbp, NULL, NULL);
        print_error(ret);
	ret = dbp->open(dbp, NULL, dbname, NULL, DB_BTREE, DB_CREATE, 0); 
	print_error(ret);
	init_DBT(&key,&data);
	ret = dbp->cursor(dbp,NULL,&cursorp,0);
	print_error(ret);
}

void init_dbb(const char *dbname)  //初始化附加索引数据库
{
        int ret;
        ret = db_create(&dbpp, NULL, 0);
	print_error(ret);
	ret = dbpp->set_flags(dbpp,DB_DUPSORT); 
	print_error(ret);
	//ret = dbp->set_dup_compare(dbp,dbcmp);
	//print_error(ret);
	ret = dbpp->set_bt_compress(dbpp, NULL, NULL);
        print_error(ret);
	ret = dbpp->open(dbpp, NULL, dbname, NULL, DB_BTREE, DB_CREATE, 0); 
	print_error(ret);
	init_DBT(&key,&data);
	ret = dbpp->cursor(dbpp,NULL,&cursorpp,0);
	print_error(ret);
}

void read_db()
{
        int ret;
        init_db(mainindexdb);
        ofstream outfile("../dbdom.txt",ofstream::out);
        string strtmp;     
        ret = cursorp->c_get(cursorp,&key,&data,DB_FIRST);
	while(ret!=DB_NOTFOUND){
	       outfile<<(char *)key.data<<" "<<((KEYNUM *)(data.data))->docid<<" "<<((KEYNUM *)(data.data))->tf;
	       ret = cursorp->c_get(cursorp,&key,&data,DB_NEXT);
        }
        outfile.close();
        if(cursorp!=NULL){
               cursorp->c_close(cursorp);
        }
        if(dbp != NULL)
    		dbp->close(dbp, 0); 
}

struct QUERYNODE{ //考虑每次询问后的排序策略
	double val;
	string doc;
	bool operator<(const QUERYNODE &cc)const{
		return val<cc.val;
	}
};

void score()
{ 
        priority_queue<QUERYNODE,vector<QUERYNODE>,less<QUERYNODE> > pq;
        //memset(mat,0,sizeof(mat));
        //memset(mmat,0,sizeof(mmat));
        memset(num,0,sizeof(num));
        long long a = GetMillSec();
        for(int i=0;i<fileind;i++){
               for(int j=0;j<keyind;j++){
               	       mat[i][j].tf = 0;
               }
        }
        for(int i=0;i<insfileind;i++){
               for(int j=0;j<keyind;j++){
               	       mmat[i][j].tf=0;
               }
        }
        for(int i=0;i<keyind;i++){
               cout<<strkey[i]<<endl;
               key.size = strkey[i].length()+1;
               key.data = (char *)strkey[i].c_str();
               int cnt = 0;
               int ret = cursorp->c_get(cursorp,&key,&data,DB_SET);
	       while(ret!=DB_NOTFOUND){ //考虑主索引数据库中该关键字的数据项
	              ++cnt;
	              printf("key = %s docid = %d tf = %d pos = %d\n ",(char *)key.data,((KEYNUM *)(data.data))->docid,((KEYNUM *)(data.data))->tf,((KEYNUM *)(data.data))->pos);
	              cout<<strfile[((KEYNUM *)(data.data))->docid]<<endl;
	              mat[((KEYNUM *)(data.data))->docid][i].tf = ((KEYNUM *)(data.data))->tf;
	              mat[((KEYNUM *)(data.data))->docid][i].pos = ((KEYNUM *)(data.data))->pos;
	              ret = cursorp->c_get(cursorp,&key,&data,DB_NEXT_DUP);
               }
               ret = cursorpp->c_get(cursorpp,&key,&data,DB_SET);
	       while(ret!=DB_NOTFOUND){ //考虑附加索引数据库中该关键字的数据项
	              ++cnt;
	              printf("key = %s docid = %d tf = %d pos = %d\n ",(char *)key.data,((KEYNUM *)(data.data))->docid,((KEYNUM *)(data.data))->tf,((KEYNUM *)(data.data))->pos);
	              cout<<insstrfile[((KEYNUM *)(data.data))->docid]<<endl;
	              mmat[((KEYNUM *)(data.data))->docid][i].tf = ((KEYNUM *)(data.data))->tf;
	              mmat[((KEYNUM *)(data.data))->docid][i].pos = ((KEYNUM *)(data.data))->pos;
	              ret = cursorpp->c_get(cursorpp,&key,&data,DB_NEXT_DUP);
               }
               num[i] = cnt;
        }
        long long b = GetMillSec();
        for(int i=0;i<fileind;i++){ //对每个原有文件进行计分
               double sum = 0.0;
               double wq = 0.0;
               for(int j=0;j<keyind;j++){
                      if(num[j]==0)
                      	     continue;
                      double wqt = log(1+1.0*(insfileind+fileind)/num[j]);
                      wq += wqt*wqt;
                      if(mat[i][j].tf){
                            double wdt = 1.0 + log(mat[i][j].tf*1.0);
                            sum += wqt*wdt;
                      }
                      cout<<mat[i][j].pos<<" "; //在某个文件中，关键字所在的位置
               }
               cout<<endl;
               if(sum<0.001)
                      continue;
               wq = sqrt(wq);
               cout<<sum<<endl;
               if(wq <0.001)
                      sum = 1e10;
               else
                      sum /= wq;
               QUERYNODE tmp;
               tmp.val = sum;
               tmp.doc = strfile[i];
               if(tmp.val>0.001)
                    cout<<tmp.doc<<endl;
               pq.push(tmp);
                    
        }      
        for(int i=0;i<insfileind;i++){ //对每个新增文件计分
               double sum = 0.0;
               double wq = 0.0;
               for(int j=0;j<keyind;j++){
                      double wqt = log(1+1.0*(insfileind+fileind)/num[j]);
                      wq += wqt*wqt;
                      if(mmat[i][j].tf){
                            double wdt = 1.0 + log(mmat[i][j].tf*1.0);
                            sum += wqt*wdt;
                      }
                      cout<<mmat[i][j].pos<<" "; //在某个文件中，关键字所在的位置
               }
               //cout<<endl;
               wq = sqrt(wq);
               if(wq <0.001)
                      sum = 1e10;
               else
                      sum /= wq;
               QUERYNODE tmp;
               tmp.val = sum;
               tmp.doc = insstrfile[i];
               pq.push(tmp);
        }    
        long long c = GetMillSec();  
        answer_ind = 0;
        int tmpsize = sizeof(keyind);
        memcpy(answer_buf+answer_ind,&keyind,tmpsize);
        //cout<<keyind<<endl;
        answer_ind += tmpsize;
        for(int i=0;i<keyind;i++){
               tmpsize = strkey[i].length()+1;
               memcpy(answer_buf+answer_ind,strkey[i].c_str(),tmpsize);
               answer_ind += tmpsize;
              // cout<<strkey[i]<<endl;
        }
        while(!pq.empty()){
              QUERYNODE tmp = pq.top();
              pq.pop();
              //cout<<tmp.doc<<tmp.val<<endl;
              if((delfile.find(tmp.doc)==delfile.end()) && (tmp.val>0.0001)){ //过滤删除文件
                    //cout<<tmp.doc<<endl;
                    //string strtmp = string((const char *)(basename((tmp.doc).c_str())));
                    char * strtmp = basename((char *)((tmp.doc).c_str()));
                    tmpsize = strlen(strtmp)+1;
                    if(answer_ind+tmpsize>9999)
                         break;
                    memcpy(answer_buf+answer_ind,strtmp,tmpsize);
                 //   cout<<strtmp<<endl;
                    answer_ind += tmpsize;
              }
        }
       /* long long d = GetMillSec();
        cout<<(b-a)*1.0/1000.0<<endl;
        cout<<(c-a)*1.0/1000.0<<endl;
        cout<<(d-a)*1.0/1000.0<<endl;*/
        //answer_buf[answer_ind*100] = 0;
}



int main()
{
        builddict();
        init_db(mainindexdb);
        init_dbb(auxindexdb);
        //read_db();
        
        struct sockaddr_in myaddr;
	int sockfd;
	if((sockfd=socket(AF_INET,SOCK_STREAM,0))<0){
		puts("socket error");
		exit(1);
	}
	myaddr.sin_family = AF_INET;
	myaddr.sin_port = htons(8080);
	myaddr.sin_addr.s_addr = INADDR_ANY;
	if(bind(sockfd,(struct sockaddr *)&myaddr,sizeof(struct sockaddr_in))==-1){
		puts("bind error");
		exit(1);
	}
	if(listen(sockfd,10)==-1){
		puts("listen error");
		exit(1);
	}
	cout<<"init search ok"<<endl;
	while(1){
		int clientfd;
		socklen_t ssize = sizeof(struct sockaddr_in);
		if((clientfd=accept(sockfd,(struct sockaddr *)&myaddr,&ssize))==-1){
			puts("accpt error");
			continue;
		}
		if(!fork()){
			long long a = GetMillSec();
		        int bytes;
		        memset(question_buf,0,sizeof(question_buf));
		        if((bytes=recv(clientfd,question_buf,100,0))==-1){
		                puts("recv error");
		                exit(1);
		        }
		        puts("asdfa");
		        puts(question_buf);
		        string inputstring = string(question_buf);
		        readfilename();
		        cout<<inputstring<<" "<<inputstring.length()<<endl;
		        long long b = GetMillSec();
	                segmentstring(inputstring);
	                long long c = GetMillSec();
	                score();
		        long long d = GetMillSec();
		        cout<<(b-a)*1.0/1000.0<<endl;
		        cout<<(c-a)*1.0/1000.0<<endl;
		        cout<<(d-a)*1.0/1000.0<<endl;
		    
			if(send(clientfd,answer_buf,answer_ind,0)==-1){
				puts("send error");
				close(clientfd);
				exit(1);
			}
		}
		close(clientfd);
	}
	if(cursorp!=NULL){
               cursorp->c_close(cursorp);
        }
        if(dbp != NULL)
    		dbp->close(dbp, 0); 
    	if(cursorpp!=NULL){
               cursorpp->c_close(cursorpp);
        }
        if(dbpp != NULL)
    		dbpp->close(dbpp, 0); 
}
